Author: Fellipe Gomes (Statistics - UFF, Data Scientist - FGV IBRE / BRAZIL)

If you liked the kernel don’t forget the upvote!

Problem Definition

This competition presents a slightly different challenge when we talking about sentiment analysis. Usually we have a dataset with some texts labeled as sentiments (negative, neutral and positive) and we develop a model that is able to estimate the sentiment of a text that is not labeled.

Here we are interested in predicting the sequence of words that best supports the feeling of the tweet (including punctuation, etc). In addition, the text can only be divided by white space. This brings up the reflection on whether it is a big deal to apply the traditional methods of cleaning and analyzing data for the application of machine learning models or whether we will have even more interesting results using alternative approaches like NER or slightly more complex approaches.

I never carried out an analysis of this in my job so before I started developing the baseline I did a quick “literature review” to find different approaches to solve this type of task and found some options like Question Answering (with BERT, XLNET, XLM, or DistilBERT) or Text summarization approache.

After submitting the baseline model, I will develop new kernels with some friends to evaluate new models using R and Python!

I hope you enjoy it!

Available data

Loading train dataset and packages used in the kernel:

# packages
library(readr)      # read/write
library(dplyr)      # manipulate data
library(tidyr)      # tidy data
library(purrr)      # functional programming
library(stringr)    # text manipulation
library(quanteda)   # textmining
library(knitr)      # print table
library(kableExtra) # custom table
library(ggplot2)    # elegant plot
library(gghalves)   # half plot
theme_set(theme_bw()) # set theme

# getting data:
train_data <- read_csv("data/train.csv")
source("functions.R")

Load functions developed in R to obtain some text metadata:

(Click " Source" to show the code)

Source

# Version 1 Functions to competition: 
# kaggle.com/c/tweet-sentiment-extraction/

#' Custom Kabble plot
#' 
#' Custom kable to kaggle kernels
#' 
#' @param x dataframe
#' @return print a custom html table
#' @example 
#' kable2(mtcars)
kable2 <- function(x, height = "300px"){
  require(knitr)
  require(kableExtra)
  
  kable(x) %>%
    kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive", "bordered"),
                  full_width = F, position = "center") %>%
    scroll_box(width = "100%", height = height)
}

#' Jaccard
#'
#' function developed to solve the problem in kaggle
#' https://www.kaggle.com/c/tweet-sentiment-extraction/
#'
#' @param str1 string
#' @param str2 string
#' @return Jaccard metric
#' @examples
#' str1 <- "Ola cOmo Vai voce"
#' str2 <- "Ola cOmo vai "
#' jaccard(str1, str2)
#'
#' str1 <- c("a a b", "a b c", "a c d")
#' str2 <- c("a d b", "a e c", "a a e")
#' purrr::map2_dbl(str1, str2, ~ jaccard(.x, .y))
jaccard <- function(str1, str2) {
  # r version for: https://www.kaggle.com/c/tweet-sentiment-extraction/overview/evaluation
  a <- unlist(strsplit(tolower(str1), split = " "))
  b <- unlist(strsplit(tolower(str2), split = " "))
  c <- intersect(a, b)
  length(c) / (length(a) + length(b) - length(c))
}

#' Clean Tweets
#'
#' fucntion to clean general text from twitter
#'
#' @param x string
#' @return cleaned string
#' @example
#' clean_text("HavE a niCe DAy!!! :)$$$")
clean_text <- function(x) {
  require(dplyr)
  require(stringr)
  
  x %>%
    str_remove_all("(RT|via)((?:\\b\\W*@\\w+)+)") %>%
    str_to_lower() %>%
    str_remove_all("@\\w+") %>%
    str_remove_all("[[:digit:]]") %>%
    str_remove_all("http(s|).*") %>%
    str_remove_all("[ |\t]{2,}") %>%
    str_remove_all("w( |)/") %>%
    str_remove_all("(?! )[^[:alnum:]]") %>%
    str_remove_all("'\\b\\w{1,2}\\b'") %>%
    str_trim() %>%
    str_squish()
}

#' Get Metadata
#'
#' function developed to get metadata of twitter dataset
#' https://www.kaggle.com/c/tweet-sentiment-extraction/data
#'
#' @param x = load dataset
#' @return dataset dataset combined with metadata
#' @example
#' metadata <- get_metadata(train_data)
#'
# metadata
get_metadata <- function(x) {
  require(dplyr)
  require(stringr)
  require(quanteda)
  require(purrr)
  t0 <- Sys.time() # to print time
  cat("Getting metadata, please wait ..\n")
  
  x <-
    x %>%
    # rename selected_text to sel_text
    rename(sel_text = selected_text) %>%
    # features engineering
    mutate(
      # on text
      text_na = is.na(text),
      text_len = str_length(text),
      text_n_words = ntoken(text),
      text_npunc = str_count(text, "[[:punct:]]"),
      text_numbers = str_count(text, "[[:digit:]]"),
      text_links = str_count(text, "http(s|).*"),
      text_hashtags = str_count(text, "#\\w+"),
      text_retweet = str_count(text, "(RT|via)((?:\\b\\W*@\\w+)+)"),
      text_atpeople = str_count(text, "@\\w+"),
      text_clean = clean_text(text),
      # on sel_text
      sel_text_na = is.na(sel_text),
      sel_text_len = str_length(sel_text),
      sel_text_n_words = ntoken(sel_text),
      sel_text_npunc = str_count(sel_text, "[[:punct:]]"),
      sel_text_numbers = str_count(sel_text, "[[:digit:]]"),
      sel_text_links = str_count(sel_text, "http(s|).*"),
      sel_text_hashtags = str_count(sel_text, "#\\w+"),
      sel_text_retweet = str_count(sel_text, "(RT|via)((?:\\b\\W*@\\w+)+)"),
      sel_text_atpeople = str_count(sel_text, "@\\w+"),
      sel_text_clean = clean_text(sel_text),
      # mutual
      equal_texts = text == sel_text,
      jaccard = purrr::map2_dbl(text, sel_text, ~ jaccard(.x, .y)),
    ) %>%
    {
      # add columns start and end sel_text
      bind_cols(.,
                map2_dfr(
                  .$text, .$sel_text,
                  ~ { # include \\ before special characters before the search
                    .y <- str_replace_all(.y, "([[:punct:]]|\\*)", "\\\\\\1")
                    str_locate(.x, .y) %>%
                      as_tibble() } ) ) } %>%
    select(textID, text, sel_text, sentiment, start, end, jaccard, 
           text_clean, sel_text_clean, everything())
  
  cat(paste0("Metadata successfully obtained!\nThe process took: ",
             round(Sys.time()-t0) ," seconds")) # Yeah!
  
  return(x)
}


See the first 100 lines of the imported dataset:

kable2(train_data[1:100,])
textID text selected_text sentiment
a3d0a7d5ad Spent the entire morning in a meeting w/ a vendor, and my boss was not happy w/ them. Lots of fun. I had other plans for my morning my boss was not happy w/ them. Lots of fun. neutral
251b6a6766 Oh! Good idea about putting them on ice cream Good positive
c9e8d1ef1c says good (or should i say bad?) afternoon! http://plurk.com/p/wxpdj says good (or should i say bad?) afternoon! neutral
f14f087215 i dont think you can vote anymore! i tried i dont think you can vote anymore! negative
bf7473b12d haha better drunken tweeting you mean? better positive
1915bebcb3 headache wanna see my Julie headache negative
2ab82634d5 had an awsome salad! I recommend getting the Spicey buffalo chicken salad! had an awsome salad! positive
a5a1c996c0 fine! Going to do my big walk today 20 or so miles fine! positive
a182b2638e Thank a yoou how are you? #TwitterTakeover Thank positive
1dcb6fdb13 Why don’t adobe realise no one WANTS to pay for Photoshop et al so they should just give it to us for free Why don’t adobe realise no one WANTS to pay for Photoshop et al so they should just give it to us for free neutral
6bff31375d PRD take a long time to review! PRD take a long time to review! neutral
e1ed43bebe _2008 Well, having to revise them! Was to do some more gardening, now, I guess hoovering and some warcraft Not played in a week ! 2008 Well, having to revise them! Was to do some more gardening, now, I guess hoovering and some warcraft Not played in a week neutral
41fd07f670 Miss you my dear Miss you my dear negative
bc0f2cb758 Have just bought a TV tuner for my laptop. He he. I deserve a present Have just bought a TV tuner for my laptop. He he. I deserve a present positive
0e6fb9153c ya mine too but for very different reason ya mine too but for very different reason neutral
49c7c9c6e2 Today Dan bought me Bio Dome AND the Reality Bites Soundtrack! Needless to say he gets MAYJAH points! Also, my tummy is not happy. Boo. , my tummy is not happy. Boo. negative
e6f5e9817d oo noo thats not good s not good negative
fee2d997a9 misses her phone… having no service sucks having no service sucks negative
459fc2a7c8 so i have like no more friends it’s kinda sad no more friends negative
1a760553fc i have perused the #fieldnotes website and it is good. too bad i must return to work i have perused the #fieldnotes website and it is good. too bad i must return to work neutral
1e857fecf5 ….welcome to public transport X ….welcome to public transport X neutral
22adf200c3 Also I popped the phone open and got all that goddamn dust out, but I wore out a clip on the camera panel so I had to glue it shut goddamn negative
bae5414377 uh oh uh oh negative
22744e80f7 12:46AM. HAppy birthday little sister of mine. Also, good night Priscilla HAppy birthday little sister of mine. positive
e487233140 aaawww no worries fresh start to work on growing it out again aaawww no worries positive
22e66a73af oh, /me still hasn’t got around to starting it oh, /me still hasn’t got around to starting it neutral
01cfa3807e is sooo tired and too busy to tweet im glad the weekend is here… yay 4 day-weekend glad positive
7f59b000cd what’s going on ? what’s going on neutral
127b2ac78c I can’t, I’m studying so I don’t fail Come over, I’ll make u some tortellini or chicken or a chicken salad. I can’t, I’m studying so I don’t fail Come over, I’ll make u some tortellini or chicken or a chicken salad. neutral
a6c77ff610 Just watched &quot;Marley &amp; Me.&quot; Cute movie. Going to watch &quot;Twilight&quot; soon! Love it! mp; Me positive
436b24fc93 not sure… there are fanciful expensive ones - but some basic ones are not expensive… Try the 24hr ‘M’ shopping place not sure… there are fanciful expensive ones - but some basic ones are not expensive. neutral
c63298701b hello thereeeeeee hello thereeeeeee neutral
4ce78f6e92 oh he is so cute… is he in uniteddogs.com? Poppy is there cute. positive
2838f92a2b has just joined the twitter community has just joined the twitter community neutral
9dd2edd0fd i’m actually starting to quite like lily allen and her music, to be honest. o quite like lily allen and her music, to be honest. positive
d86eaa0487 what Brody how dare u what Brody how dare u negative
2756df3899 I feel useless I don’t know what to do right now. I’m so bored useless negative
daadcab8af I’m sooo HAPPY Demi’s back on twitter! I’m sooo HAPPY De positive
2334cae701 kate is leaving me all by my lonesome lonesome negative
89fc963188 I was rooting for Betty. I was rooting for Betty. neutral
ebe46589f3 That would be most welcome That would be most welcome positive
7f0c862bad Why do weddings on TV make me cry? Is it just my inner desire to be eternally happy and start the rest of my life?? happy positive
b63f6f0bf9 Is hungry with a sub Is hungry with a sub neutral
118850b843 I lost a follower oh well probably one of those mute ones anyway. lost negative
2d5648ba3c i see you there i see you there neutral
ad6c3de0ae saying goodbye to something familiar never becomes familiar never becomes familiar negative
791449f114 I can’t comment directly to someone…… and I’m here all alone I can’t comment directly to someone…… and I’m here all alone neutral
4f26fc0981 aw, that’s cute aw, that’s cute positive
95c725edb6 Just got back in, and I’m absolutely exhausted, have to be up in 5 hours for work I’m absolutely exhausted, negative
fe98872319 I saw them a few on this tour a few months ago in Boston. So good! Glad you finally got to see them So good! Glad you finally got to see them positive
5498496af0 after 4 years of trying, my cat today figured out how to open the under-the-sink cupboard. time for a target run to buy safety locks after 4 years of trying, my cat today figured out how to open the under-the-sink cupboard. time for a target run to buy safety locks neutral
d11234ed68 Heading to the U of Utah hospital. Have a great day everyone Have a great day everyone positive
e6dedf3aad my dog ran awayyy my dog ran awayyy negative
556f187ce7 omggg, how were the &quot;crazy cobras&quot; i wantewd to see them when they were in London omggg, how were the &quot;crazy cobras&quot; i wantewd to see them when they were neutral
0bc6d79c40 Today seems like it should be a good day! Even though my fuel pump is going out on my car xoxo Today seems like it should be a good day! positive
384c73b996 It’s Raining! It’s Raining! neutral
e41986b2d9 is missing someone.. haayy.. http://plurk.com/p/x2avn is missing someone.. negative
3440297f8b S’ok, trying to plot alternatives as we speak sigh sigh negative
f09c712326 Hot ****, I’m at **** Disneyland! Hot ****, negative
acdea0cf39 Just figured out why only strong guys are supposed to put heavy things on top shelves, ouch! ouch! negative
675bb05b9d Im @ the dentist ….scary people here… .scary negative
97bc17f4b4 waiting for my mom to get home waiting for my mom to get home neutral
0338930089 so most kids love my dark hair some say that I dye it too much and they never know what color its going to be yea me hahahahaha love positive
07836035cd damm i feel like the song of dead and gone by Travis Garland damm negative
dfc5afdab4 UGH… I am soooo drag assing today but on a brighter note I got my G1 update &amp; it is the biznesssss! UGH… I am soooo drag assing today but on a brighter note I got my G1 update &amp; it is the biznesss neutral
b5167d5a96 Feels like im going to cough up a lung Feels like im going to cough up a lung negative
060d8bb5d5 actually, I wish I was back in Tahoe. I miss it there. I miss it there negative
3d37d0aa09 sounds like you’ve had a nice weekend lady! nice positive
021e4d7fd7 See you on may 16!! See you on may 16!! neutral
4ffb09c454 _hall no i was gutted when he wasn’t. lmao. i think i’m obsessed with him, bahaha. hall no i was gutted when he wasn’t. lmao. i think i’m obsessed with him, bahaha. positive
f29df52d91 I want to know when the auditions are Mander! Text or…reply please! I want to know when the auditions are Mander! Text or…reply please! neutral
196107983d I’ll stop by for a bit when I’m off. But have to rush to rehearsal - no quiet for me tonight. I’ll stop by for a bit when I’m off. But have to rush to rehearsal - no quiet for me tonight. neutral
2f2542a3c2 Whoa, I didn’t mean to sleep for more than an hour Whoa, I didn’t mean to sleep for more than an hour neutral
134b4e3267 In bed after a good last saturday in ec. I’m gonna miss this place but can’t wait til summer In bed after a good last saturday in ec. I’m gonna miss this place but can’t wait til summer negative
5ac1a82457 Is the Bee to Busy to talk to his buddy on Yahoo? Is the Bee to Busy to talk to his buddy on Yahoo? neutral
aaee08f683 I need followers!!! I need followers!!! neutral
ba5468c0c3 Im on my way to brookings Im on my way to brookings neutral
269e7185cb _44 Thanks!! It was a bit out of my comfort zone but it was fun _44 Thanks!! It was a bit out of my comfort zone but it was fun neutral
193909d380 there are many jobs I aspire to, supervisor is not one of them talk about too much stress for too little pay stress negative
ba8aa5e888 is watching the canucks is watching the canucks neutral
b4d9e63784 Thats ok…ur a good person + My Idol soo good people deserve sum good comments..hows the weather ova der –London is cold good person positive
dfeadc340d blerg, **** you Monday, why must you always come. Decent weekend, new glasses, hair color and got carded at Gamestop = awesome ego boost blerg, **** you Monday, why must you always come. Decent weekend, new glasses, hair color and got carded at Gamestop = awesome ego boost neutral
f4f97d8289 In children’s hospital ER hoping Meredith does NOT have a broken elbow/arm hoping positive
b2fed31eb7 gprof problem on Mac why ?? gprof problem on Mac why ?? neutral
6903cb08f2 Nice to see you tweeting! It’s Sunday 10th May and we’re celebrating Mother’s Day here today. So be nice to yer Mom e nice positive
7a0f49dddd Yay! You finally get to have a trip away from home again! How exciting! You 2 going to shop a bit ? ! How exciting! positive
b2986c944a done with the fkn exam! n just bought the mothers day gift, finally… tada! the new prada fragrance going to meet up mum n fam now : … done with the fkn exam! n just bought the mothers day gift, finally… tada! the new prada fragrance going to meet up mum n fam now : … neutral
653cf8e5a6 just got home from a partyyyy. had a good time. can’t wait for my birthday in 7 days! had a good time. positive
514174445d Goodnight Beautiful world sweet dreams. olive juice! Goodnight Beautiful world sweet dreams. olive juice! positive
88c05add09 _Gamefan Thanks I love &quot;Misery Business&quot;; Paramore is such a great band! love positive
eaa0b85855 Yay for St Louis traffic Yay for St Louis traffic neutral
ab56747ae1 I have to pee. I have to pee. neutral
b57188597f hhahaa! its well strange just woke up to an empty house ahah lolllllyyyyyyyy&lt;3 love that dog hhahaa! its well strange just woke up to an empty house ahah lolllllyyyyyyyy&lt;3 love th neutral
7e4579c8f9 awesome! I am at 1505 west lake street: right across from the dunn brothers on lake awesome! positive
f20fecf55e i got followed by one that says she doesn’t talk to &quot;god people&quot;-i wonder if she talks to &quot;goddess people?&quot; i got followed by one that says she doesn’t talk to &quot;god people&quot;-i wonder if she talks to &quot;god neutral
f6ca88db6f http://twitpic.com/66r0u - uh wait…i thought this was the 29th…tonight uh wait…i thought this was the 29th…tonight neutral
067058825c Will this rain ever go away, my puppy is getting cabin fever and she doesn’t like going out in the rain doesn’t like negative
b084bce92a Great meeting…lunch on the patio in the sunshine…nothing more perfect busy day…time for bed, ahhhhh Great meeting.. positive
2ef8bef8cf &quot;We did it we did it oh yea yea yea no eating here tonite woo eating here tonight no no no eating here tonite you on a diet&quot; finding nemo &quot;We did it we did it oh yea yea yea no eating here tonite woo eating here tonight no no no eating here tonite you on a diet&quot; findi neutral
aa4412f02c hey adt, guess what. my princelple’s number plate is adt�000 well its not 000, i just dunno the numbers. hey adt, guess what. my princelple’s number plate is adt�000 well its not 000, i just dunno the numbers. neutral

The training dataset provided has 27486 rows and 4 columns.

Getting Metadata

Use the function get_metadata() developed for feature engineering of this dataset (twitter texts):

train_metadata <- get_metadata(train_data)
## Getting metadata, please wait ..
## Metadata successfully obtained!
## The process took: 40 seconds
kable2(train_metadata[1:100,])
textID text sel_text sentiment start end jaccard text_clean sel_text_clean text_na text_len text_n_words text_npunc text_numbers text_links text_hashtags text_retweet text_atpeople sel_text_na sel_text_len sel_text_n_words sel_text_npunc sel_text_numbers sel_text_links sel_text_hashtags sel_text_retweet sel_text_atpeople equal_texts
a3d0a7d5ad Spent the entire morning in a meeting w/ a vendor, and my boss was not happy w/ them. Lots of fun. I had other plans for my morning my boss was not happy w/ them. Lots of fun. neutral 56 98 0.3448276 spent the entire morning in a meeting a vendor and my boss was not happy them lots of funi had other plans for my morning my boss was not happy them lots of fun FALSE 132 33 5 0 0 0 0 0 FALSE 43 13 3 0 0 0 0 0 FALSE
251b6a6766 Oh! Good idea about putting them on ice cream Good positive 5 8 0.1111111 oh good idea about putting them on ice cream good FALSE 45 10 1 0 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
c9e8d1ef1c says good (or should i say bad?) afternoon! http://plurk.com/p/wxpdj says good (or should i say bad?) afternoon! neutral 1 43 0.8000000 says good or should i say bad afternoon says good or should i say bad afternoon FALSE 69 13 10 0 1 0 0 0 FALSE 43 12 4 0 0 0 0 0 FALSE
f14f087215 i dont think you can vote anymore! i tried i dont think you can vote anymore! negative 1 34 0.7777778 i dont think you can vote anymore i tried i dont think you can vote anymore FALSE 42 10 1 0 0 0 0 0 FALSE 34 8 1 0 0 0 0 0 FALSE
bf7473b12d haha better drunken tweeting you mean? better positive 6 11 0.1666667 haha better drunken tweeting you mean better FALSE 38 7 1 0 0 0 0 0 FALSE 6 1 0 0 0 0 0 0 FALSE
1915bebcb3 headache wanna see my Julie headache negative 1 8 0.1666667 headachewanna see my julie headache FALSE 28 5 0 0 0 0 0 0 FALSE 8 1 0 0 0 0 0 0 FALSE
2ab82634d5 had an awsome salad! I recommend getting the Spicey buffalo chicken salad! had an awsome salad! positive 1 20 0.3333333 had an awsome salad i recommend getting the spicey buffalo chicken salad had an awsome salad FALSE 74 14 2 0 0 0 0 0 FALSE 20 5 1 0 0 0 0 0 FALSE
a5a1c996c0 fine! Going to do my big walk today 20 or so miles fine! positive 1 5 0.0833333 fine going to do my big walk todayor so miles fine FALSE 50 13 1 2 0 0 0 0 FALSE 5 2 1 0 0 0 0 0 FALSE
a182b2638e Thank a yoou how are you? #TwitterTakeover Thank positive 1 5 0.1250000 thank a yoouhow are you twittertakeover thank FALSE 43 8 2 0 0 1 0 0 FALSE 5 1 0 0 0 0 0 0 FALSE
1dcb6fdb13 Why don’t adobe realise no one WANTS to pay for Photoshop et al so they should just give it to us for free Why don’t adobe realise no one WANTS to pay for Photoshop et al so they should just give it to us for free neutral 1 106 0.8400000 why dont adobe realise no one wants to pay for photoshop et al so they should just give it to us for free why dont adobe realise no one wants to pay for photoshop et al so they should just give it to us for free FALSE 106 23 1 0 0 0 0 0 FALSE 106 23 1 0 0 0 0 0 TRUE
6bff31375d PRD take a long time to review! PRD take a long time to review! neutral 1 31 1.0000000 prd take a long time to review prd take a long time to review FALSE 31 8 1 0 0 0 0 0 FALSE 31 8 1 0 0 0 0 0 TRUE
e1ed43bebe _2008 Well, having to revise them! Was to do some more gardening, now, I guess hoovering and some warcraft Not played in a week ! 2008 Well, having to revise them! Was to do some more gardening, now, I guess hoovering and some warcraft Not played in a week neutral 2 129 0.7096774 well having to revise themwas to do some more gardening now i guess hoovering and some warcraftnot played in a week well having to revise themwas to do some more gardening now i guess hoovering and some warcraftnot played in a week FALSE 131 29 6 4 0 0 0 0 FALSE 128 28 4 4 0 0 0 0 FALSE
41fd07f670 Miss you my dear Miss you my dear negative 1 16 1.0000000 miss you my dear miss you my dear FALSE 16 4 0 0 0 0 0 0 FALSE 16 4 0 0 0 0 0 0 TRUE
bc0f2cb758 Have just bought a TV tuner for my laptop. He he. I deserve a present Have just bought a TV tuner for my laptop. He he. I deserve a present positive 1 71 0.7894737 have just bought a tv tuner for my laptophe hei deserve a present have just bought a tv tuner for my laptophe hei deserve a present FALSE 71 17 2 0 0 0 0 0 FALSE 71 17 2 0 0 0 0 0 TRUE
0e6fb9153c ya mine too but for very different reason ya mine too but for very different reason neutral 1 41 1.0000000 ya mine too but for very different reason ya mine too but for very different reason FALSE 41 8 0 0 0 0 0 0 FALSE 41 8 0 0 0 0 0 0 TRUE
49c7c9c6e2 Today Dan bought me Bio Dome AND the Reality Bites Soundtrack! Needless to say he gets MAYJAH points! Also, my tummy is not happy. Boo. , my tummy is not happy. Boo. negative 108 136 0.2222222 today dan bought me bio dome and the reality bites soundtrack needless to say he gets mayjah pointsalso my tummy is not happy boo my tummy is not happy boo FALSE 136 30 5 0 0 0 0 0 FALSE 29 9 3 0 0 0 0 0 FALSE
e6f5e9817d oo noo thats not good s not good negative 12 21 0.3333333 oo noo thats not good s not good FALSE 21 5 0 0 0 0 0 0 FALSE 10 3 0 0 0 0 0 0 FALSE
fee2d997a9 misses her phone… having no service sucks having no service sucks negative 21 43 0.5714286 misses her phone having no service sucks having no service sucks FALSE 43 10 3 0 0 0 0 0 FALSE 23 4 0 0 0 0 0 0 FALSE
459fc2a7c8 so i have like no more friends it’s kinda sad no more friends negative 16 30 0.3000000 so i have like no more friends its kinda sad no more friends FALSE 45 10 1 0 0 0 0 0 FALSE 15 3 0 0 0 0 0 0 FALSE
1a760553fc i have perused the #fieldnotes website and it is good. too bad i must return to work i have perused the #fieldnotes website and it is good. too bad i must return to work neutral 1 85 0.8947368 i have perused the fieldnotes website and it is goodtoo bad i must return to work i have perused the fieldnotes website and it is goodtoo bad i must return to work FALSE 85 18 2 0 0 1 0 0 FALSE 85 18 2 0 0 1 0 0 TRUE
1e857fecf5 ….welcome to public transport X ….welcome to public transport X neutral 1 34 1.0000000 welcome to public transportx welcome to public transportx FALSE 34 9 4 0 0 0 0 0 FALSE 34 9 4 0 0 0 0 0 TRUE
22adf200c3 Also I popped the phone open and got all that goddamn dust out, but I wore out a clip on the camera panel so I had to glue it shut goddamn negative 47 53 0.0333333 also i popped the phone open and got all that goddamn dust out but i wore out a clip on the camera panel so i had to glue it shut goddamn FALSE 130 31 1 0 0 0 0 0 FALSE 7 1 0 0 0 0 0 0 FALSE
bae5414377 uh oh uh oh negative 1 5 1.0000000 uh oh uh oh FALSE 5 2 0 0 0 0 0 0 FALSE 5 2 0 0 0 0 0 0 TRUE
22744e80f7 12:46AM. HAppy birthday little sister of mine. Also, good night Priscilla HAppy birthday little sister of mine. positive 10 46 0.5454545 am happy birthday little sister of mine also good night priscilla happy birthday little sister of mine FALSE 73 16 4 4 0 0 0 0 FALSE 37 7 1 0 0 0 0 0 FALSE
e487233140 aaawww no worries fresh start to work on growing it out again aaawww no worries positive 1 18 0.3076923 aaawwwno worries fresh start to work on growing it out again aaawwwno worries FALSE 62 12 0 0 0 0 0 0 FALSE 18 3 0 0 0 0 0 0 FALSE
22e66a73af oh, /me still hasn’t got around to starting it oh, /me still hasn’t got around to starting it neutral 1 46 1.0000000 oh me still hasnt got around to starting it oh me still hasnt got around to starting it FALSE 46 11 3 0 0 0 0 0 FALSE 46 11 3 0 0 0 0 0 TRUE
01cfa3807e is sooo tired and too busy to tweet im glad the weekend is here… yay 4 day-weekend glad positive 41 44 0.0555556 is sooo tired and too busy to tweetim glad the weekend is here yaydayweekend glad FALSE 85 20 4 1 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
7f59b000cd what’s going on ? what’s going on neutral 1 15 0.7500000 whats going on whats going on FALSE 17 4 2 0 0 0 0 0 FALSE 15 3 1 0 0 0 0 0 FALSE
127b2ac78c I can’t, I’m studying so I don’t fail Come over, I’ll make u some tortellini or chicken or a chicken salad. I can’t, I’m studying so I don’t fail Come over, I’ll make u some tortellini or chicken or a chicken salad. neutral 1 108 0.7600000 i cant im studying so i dont failcome over ill make u some tortellini or chicken or a chicken salad i cant im studying so i dont failcome over ill make u some tortellini or chicken or a chicken salad FALSE 108 24 7 0 0 0 0 0 FALSE 108 24 7 0 0 0 0 0 TRUE
a6c77ff610 Just watched &quot;Marley &amp; Me.&quot; Cute movie. Going to watch &quot;Twilight&quot; soon! Love it! mp; Me positive 29 34 0.0000000 just watched quotmarley amp mequot cute movie going to watch quottwilightquot soonlove it mp me FALSE 105 32 14 0 0 0 0 0 FALSE 6 3 1 0 0 0 0 0 FALSE
436b24fc93 not sure… there are fanciful expensive ones - but some basic ones are not expensive… Try the 24hr ‘M’ shopping place not sure… there are fanciful expensive ones - but some basic ones are not expensive. neutral 1 86 0.4400000 not sure there are fanciful expensive ones but some basic ones are not expensive try the hr m shopping place not sure there are fanciful expensive ones but some basic ones are not expensive FALSE 120 29 9 2 0 0 0 0 FALSE 86 19 5 0 0 0 0 0 FALSE
c63298701b hello thereeeeeee hello thereeeeeee neutral 1 17 1.0000000 hello thereeeeeee hello thereeeeeee FALSE 17 2 0 0 0 0 0 0 FALSE 17 2 0 0 0 0 0 0 TRUE
4ce78f6e92 oh he is so cute… is he in uniteddogs.com? Poppy is there cute. positive 13 17 0.0000000 oh he is so cute is he in uniteddogscom poppy is there cute FALSE 59 16 5 0 0 0 0 0 FALSE 5 2 1 0 0 0 0 0 FALSE
2838f92a2b has just joined the twitter community has just joined the twitter community neutral 1 37 1.0000000 has just joined the twitter community has just joined the twitter community FALSE 37 6 0 0 0 0 0 0 FALSE 37 6 0 0 0 0 0 0 TRUE
9dd2edd0fd i’m actually starting to quite like lily allen and her music, to be honest. o quite like lily allen and her music, to be honest. positive 24 75 0.6666667 im actually starting to quite like lily allen and her music to be honest o quite like lily allen and her music to be honest FALSE 75 16 3 0 0 0 0 0 FALSE 52 13 2 0 0 0 0 0 FALSE
d86eaa0487 what Brody how dare u what Brody how dare u negative 1 21 1.0000000 what brody how dare u what brody how dare u FALSE 21 5 0 0 0 0 0 0 FALSE 21 5 0 0 0 0 0 0 TRUE
2756df3899 I feel useless I don’t know what to do right now. I’m so bored useless negative 8 14 0.0714286 i feel useless i dont know what to do right now im so bored useless FALSE 62 15 3 0 0 0 0 0 FALSE 7 1 0 0 0 0 0 0 FALSE
daadcab8af I’m sooo HAPPY Demi’s back on twitter! I’m sooo HAPPY De positive 1 17 0.3750000 im sooo happy demis back on twitter im sooo happy de FALSE 38 8 3 0 0 0 0 0 FALSE 17 4 1 0 0 0 0 0 FALSE
2334cae701 kate is leaving me all by my lonesome lonesome negative 30 37 0.1250000 kate is leaving me all by my lonesome lonesome FALSE 37 8 0 0 0 0 0 0 FALSE 8 1 0 0 0 0 0 0 FALSE
89fc963188 I was rooting for Betty. I was rooting for Betty. neutral 1 24 1.0000000 i was rooting for betty i was rooting for betty FALSE 24 6 1 0 0 0 0 0 FALSE 24 6 1 0 0 0 0 0 TRUE
ebe46589f3 That would be most welcome That would be most welcome positive 1 26 1.0000000 that would be most welcome that would be most welcome FALSE 26 5 0 0 0 0 0 0 FALSE 26 5 0 0 0 0 0 0 TRUE
7f0c862bad Why do weddings on TV make me cry? Is it just my inner desire to be eternally happy and start the rest of my life?? happy positive 79 83 0.0400000 why do weddings on tv make me cry is it just my inner desire to be eternally happy and start the rest of my life happy FALSE 115 28 3 0 0 0 0 0 FALSE 5 1 0 0 0 0 0 0 FALSE
b63f6f0bf9 Is hungry with a sub Is hungry with a sub neutral 1 20 1.0000000 is hungry with a sub is hungry with a sub FALSE 20 5 0 0 0 0 0 0 FALSE 20 5 0 0 0 0 0 0 TRUE
118850b843 I lost a follower oh well probably one of those mute ones anyway. lost negative 3 6 0.0714286 i lost a followeroh well probably one of those mute ones anyway lost FALSE 66 14 1 0 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
2d5648ba3c i see you there i see you there neutral 1 15 1.0000000 i see you there i see you there FALSE 15 4 0 0 0 0 0 0 FALSE 15 4 0 0 0 0 0 0 TRUE
ad6c3de0ae saying goodbye to something familiar never becomes familiar never becomes familiar negative 38 59 0.3750000 saying goodbye to something familiar never becomes familiar never becomes familiar FALSE 59 8 0 0 0 0 0 0 FALSE 22 3 0 0 0 0 0 0 FALSE
791449f114 I can’t comment directly to someone…… and I’m here all alone I can’t comment directly to someone…… and I’m here all alone neutral 1 64 1.0000000 i cant comment directly to someone and im here all alone i cant comment directly to someone and im here all alone FALSE 64 17 8 0 0 0 0 0 FALSE 64 17 8 0 0 0 0 0 TRUE
4f26fc0981 aw, that’s cute aw, that’s cute positive 1 15 1.0000000 aw thats cute aw thats cute FALSE 15 4 2 0 0 0 0 0 FALSE 15 4 2 0 0 0 0 0 TRUE
95c725edb6 Just got back in, and I’m absolutely exhausted, have to be up in 5 hours for work I’m absolutely exhausted, negative 23 47 0.1764706 just got back in and im absolutely exhausted have to be up inhours for work im absolutely exhausted FALSE 81 19 3 1 0 0 0 0 FALSE 25 4 2 0 0 0 0 0 FALSE
fe98872319 I saw them a few on this tour a few months ago in Boston. So good! Glad you finally got to see them So good! Glad you finally got to see them positive 59 99 0.3913043 i saw them a few on this tour a few months ago in boston so good glad you finally got to see them so good glad you finally got to see them FALSE 99 25 2 0 0 0 0 0 FALSE 41 10 1 0 0 0 0 0 FALSE
5498496af0 after 4 years of trying, my cat today figured out how to open the under-the-sink cupboard. time for a target run to buy safety locks after 4 years of trying, my cat today figured out how to open the under-the-sink cupboard. time for a target run to buy safety locks neutral 1 133 0.9259259 afteryears of trying my cat today figured out how to open the underthesink cupboardtime for a target run to buy safety locks afteryears of trying my cat today figured out how to open the underthesink cupboardtime for a target run to buy safety locks FALSE 133 27 4 1 0 0 0 0 FALSE 133 27 4 1 0 0 0 0 TRUE
d11234ed68 Heading to the U of Utah hospital. Have a great day everyone Have a great day everyone positive 36 60 0.4166667 heading to the u of utah hospital have a great day everyone have a great day everyone FALSE 60 13 1 0 0 0 0 0 FALSE 25 5 0 0 0 0 0 0 FALSE
e6dedf3aad my dog ran awayyy my dog ran awayyy negative 1 17 1.0000000 my dog ran awayyy my dog ran awayyy FALSE 17 4 0 0 0 0 0 0 FALSE 17 4 0 0 0 0 0 0 TRUE
556f187ce7 omggg, how were the &quot;crazy cobras&quot; i wantewd to see them when they were in London omggg, how were the &quot;crazy cobras&quot; i wantewd to see them when they were neutral 1 81 0.7647059 omggg how were the quotcrazy cobrasquot i wantewd to see them when they were in london omggg how were the quotcrazy cobrasquot i wantewd to see them when they were FALSE 91 23 5 0 0 0 0 0 FALSE 81 21 5 0 0 0 0 0 FALSE
0bc6d79c40 Today seems like it should be a good day! Even though my fuel pump is going out on my car xoxo Today seems like it should be a good day! positive 1 41 0.4090909 today seems like it should be a good day even though my fuel pump is going out on my carxoxo today seems like it should be a good day FALSE 99 26 3 0 0 0 0 0 FALSE 41 10 1 0 0 0 0 0 FALSE
384c73b996 It’s Raining! It’s Raining! neutral 1 13 1.0000000 its raining its raining FALSE 13 3 2 0 0 0 0 0 FALSE 13 3 2 0 0 0 0 0 TRUE
e41986b2d9 is missing someone.. haayy.. http://plurk.com/p/x2avn is missing someone.. negative 1 20 0.5000000 is missing someone haayy is missing someone FALSE 54 9 10 1 1 0 0 0 FALSE 20 5 2 0 0 0 0 0 FALSE
3440297f8b S’ok, trying to plot alternatives as we speak sigh sigh negative 47 52 0.1111111 sok trying to plot alternatives as we speak sigh sigh FALSE 52 12 4 0 0 0 0 0 FALSE 6 3 2 0 0 0 0 0 FALSE
f09c712326 Hot ****, I’m at **** Disneyland! Hot ****, negative 1 9 0.3333333 hot im at disneyland hot FALSE 33 14 11 0 0 0 0 0 FALSE 9 6 5 0 0 0 0 0 FALSE
acdea0cf39 Just figured out why only strong guys are supposed to put heavy things on top shelves, ouch! ouch! negative 88 92 0.0588235 just figured out why only strong guys are supposed to put heavy things on top shelves ouch ouch FALSE 92 19 2 0 0 0 0 0 FALSE 5 2 1 0 0 0 0 0 FALSE
675bb05b9d Im @ the dentist ….scary people here… .scary negative 22 27 0.0000000 im the dentistscary people here scary FALSE 42 14 8 0 0 0 0 0 FALSE 6 2 1 0 0 0 0 0 FALSE
97bc17f4b4 waiting for my mom to get home waiting for my mom to get home neutral 1 30 1.0000000 waiting for my mom to get home waiting for my mom to get home FALSE 30 7 0 0 0 0 0 0 FALSE 30 7 0 0 0 0 0 0 TRUE
0338930089 so most kids love my dark hair some say that I dye it too much and they never know what color its going to be yea me hahahahaha love positive 14 17 0.0344828 so most kids love my dark hair some say that i dye it too much and they never know what color its going to beyea me hahahahaha love FALSE 128 28 0 0 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
07836035cd damm i feel like the song of dead and gone by Travis Garland damm negative 1 4 0.0769231 damm i feel like the song of dead and gone by travis garland damm FALSE 60 13 0 0 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
dfc5afdab4 UGH… I am soooo drag assing today but on a brighter note I got my G1 update &amp; it is the biznesssss! UGH… I am soooo drag assing today but on a brighter note I got my G1 update &amp; it is the biznesss neutral 1 103 0.8400000 ugh i am soooo drag assing todaybut on a brighter note i got my g update amp it is the biznesssss ugh i am soooo drag assing todaybut on a brighter note i got my g update amp it is the biznesss FALSE 106 28 6 1 0 0 0 0 FALSE 103 27 5 1 0 0 0 0 FALSE
b5167d5a96 Feels like im going to cough up a lung Feels like im going to cough up a lung negative 1 38 1.0000000 feels like im going to cough up a lung feels like im going to cough up a lung FALSE 38 9 0 0 0 0 0 0 FALSE 38 9 0 0 0 0 0 0 TRUE
060d8bb5d5 actually, I wish I was back in Tahoe. I miss it there. I miss it there negative 40 54 0.2142857 actually i wish i was back in tahoei miss it there i miss it there FALSE 55 15 3 0 0 0 0 0 FALSE 15 4 0 0 0 0 0 0 FALSE
3d37d0aa09 sounds like you’ve had a nice weekend lady! nice positive 26 29 0.1250000 sounds like youve had a nice weekend lady nice FALSE 43 9 2 0 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
021e4d7fd7 See you on may 16!! See you on may 16!! neutral 1 19 1.0000000 see you on may see you on may FALSE 19 7 2 2 0 0 0 0 FALSE 19 7 2 2 0 0 0 0 TRUE
4ffb09c454 _hall no i was gutted when he wasn’t. lmao. i think i’m obsessed with him, bahaha. hall no i was gutted when he wasn’t. lmao. i think i’m obsessed with him, bahaha. positive 2 83 0.7894737 hall noi was gutted when he wasnt lmao i think im obsessed with him bahaha hall noi was gutted when he wasnt lmao i think im obsessed with him bahaha FALSE 83 20 7 0 0 0 0 0 FALSE 82 20 6 0 0 0 0 0 FALSE
f29df52d91 I want to know when the auditions are Mander! Text or…reply please! I want to know when the auditions are Mander! Text or…reply please! neutral 1 69 1.0000000 i want to know when the auditions are mander text orreply please i want to know when the auditions are mander text orreply please FALSE 69 18 5 0 0 0 0 0 FALSE 69 18 5 0 0 0 0 0 TRUE
196107983d I’ll stop by for a bit when I’m off. But have to rush to rehearsal - no quiet for me tonight. I’ll stop by for a bit when I’m off. But have to rush to rehearsal - no quiet for me tonight. neutral 1 94 0.8333333 ill stop by for a bit when im offbut have to rush to rehearsal no quiet for me tonight ill stop by for a bit when im offbut have to rush to rehearsal no quiet for me tonight FALSE 94 23 5 0 0 0 0 0 FALSE 94 23 5 0 0 0 0 0 TRUE
2f2542a3c2 Whoa, I didn’t mean to sleep for more than an hour Whoa, I didn’t mean to sleep for more than an hour neutral 1 50 1.0000000 whoa i didnt mean to sleep for more than an hour whoa i didnt mean to sleep for more than an hour FALSE 50 12 2 0 0 0 0 0 FALSE 50 12 2 0 0 0 0 0 TRUE
134b4e3267 In bed after a good last saturday in ec. I’m gonna miss this place but can’t wait til summer In bed after a good last saturday in ec. I’m gonna miss this place but can’t wait til summer negative 1 92 0.9000000 in bed after a good last saturday in ec im gonna miss this place but cant wait til summer in bed after a good last saturday in ec im gonna miss this place but cant wait til summer FALSE 92 20 3 0 0 0 0 0 FALSE 92 20 3 0 0 0 0 0 TRUE
5ac1a82457 Is the Bee to Busy to talk to his buddy on Yahoo? Is the Bee to Busy to talk to his buddy on Yahoo? neutral 1 49 0.7142857 is the bee to busy to talk to his buddy on yahoo is the bee to busy to talk to his buddy on yahoo FALSE 49 13 1 0 0 0 0 0 FALSE 49 13 1 0 0 0 0 0 TRUE
aaee08f683 I need followers!!! I need followers!!! neutral 1 19 1.0000000 i need followers i need followers FALSE 19 6 3 0 0 0 0 0 FALSE 19 6 3 0 0 0 0 0 TRUE
ba5468c0c3 Im on my way to brookings Im on my way to brookings neutral 1 25 1.0000000 im on my way to brookings im on my way to brookings FALSE 25 6 0 0 0 0 0 0 FALSE 25 6 0 0 0 0 0 0 TRUE
269e7185cb _44 Thanks!! It was a bit out of my comfort zone but it was fun _44 Thanks!! It was a bit out of my comfort zone but it was fun neutral 1 63 0.7647059 thanks it was a bit out of my comfort zone but it was fun thanks it was a bit out of my comfort zone but it was fun FALSE 63 17 3 2 0 0 0 0 FALSE 63 17 3 2 0 0 0 0 TRUE
193909d380 there are many jobs I aspire to, supervisor is not one of them talk about too much stress for too little pay stress negative 85 90 0.0434783 there are many jobs i aspire to supervisor is not one of themtalk about too much stress for too little pay stress FALSE 109 23 1 0 0 0 0 0 FALSE 6 1 0 0 0 0 0 0 FALSE
ba8aa5e888 is watching the canucks is watching the canucks neutral 1 23 1.0000000 is watching the canucks is watching the canucks FALSE 23 4 0 0 0 0 0 0 FALSE 23 4 0 0 0 0 0 0 TRUE
b4d9e63784 Thats ok…ur a good person + My Idol soo good people deserve sum good comments..hows the weather ova der –London is cold good person positive 17 27 0.0909091 thats okur a good person my idol soo good people deserve sum good commentshows the weather ova der london is cold good person FALSE 122 30 7 0 0 0 0 0 FALSE 11 2 0 0 0 0 0 0 FALSE
dfeadc340d blerg, **** you Monday, why must you always come. Decent weekend, new glasses, hair color and got carded at Gamestop = awesome ego boost blerg, **** you Monday, why must you always come. Decent weekend, new glasses, hair color and got carded at Gamestop = awesome ego boost neutral 1 137 0.9230769 blerg you monday why must you always comedecent weekend new glasses hair color and got carded at gamestop awesome ego boost blerg you monday why must you always comedecent weekend new glasses hair color and got carded at gamestop awesome ego boost FALSE 137 32 9 0 0 0 0 0 FALSE 137 32 9 0 0 0 0 0 TRUE
f4f97d8289 In children’s hospital ER hoping Meredith does NOT have a broken elbow/arm hoping positive 27 32 0.0833333 in childrens hospital er hoping meredith does not have a broken elboarm hoping FALSE 74 14 2 0 0 0 0 0 FALSE 6 1 0 0 0 0 0 0 FALSE
b2fed31eb7 gprof problem on Mac why ?? gprof problem on Mac why ?? neutral 1 28 1.0000000 gprof problem on macwhy gprof problem on macwhy FALSE 28 7 2 0 0 0 0 0 FALSE 28 7 2 0 0 0 0 0 TRUE
6903cb08f2 Nice to see you tweeting! It’s Sunday 10th May and we’re celebrating Mother’s Day here today. So be nice to yer Mom e nice positive 100 105 0.0416667 nice to see you tweetingits sunday th may and were celebrating mothers day here today so be nice to yer mom e nice FALSE 116 24 5 2 0 0 0 0 FALSE 6 2 0 0 0 0 0 0 FALSE
7a0f49dddd Yay! You finally get to have a trip away from home again! How exciting! You 2 going to shop a bit ? ! How exciting! positive 57 71 0.0869565 yay you finally get to have a trip away from home again how exciting yougoing to shop a bit how exciting FALSE 99 25 4 1 0 0 0 0 FALSE 15 4 2 0 0 0 0 0 FALSE
b2986c944a done with the fkn exam! n just bought the mothers day gift, finally… tada! the new prada fragrance going to meet up mum n fam now : … done with the fkn exam! n just bought the mothers day gift, finally… tada! the new prada fragrance going to meet up mum n fam now : … neutral 1 138 0.8125000 done with the fkn exam n just bought the mothers day gift finally tada the new prada fragrancegoing to meet up mum n fam now done with the fkn exam n just bought the mothers day gift finally tada the new prada fragrancegoing to meet up mum n fam now FALSE 138 36 10 0 0 0 0 0 FALSE 138 36 10 0 0 0 0 0 TRUE
653cf8e5a6 just got home from a partyyyy. had a good time. can’t wait for my birthday in 7 days! had a good time. positive 32 47 0.2222222 just got home from a partyyyy had a good time cant wait for my birthday indays had a good time FALSE 85 21 4 1 0 0 0 0 FALSE 16 5 1 0 0 0 0 0 FALSE
514174445d Goodnight Beautiful world sweet dreams. olive juice! Goodnight Beautiful world sweet dreams. olive juice! positive 1 53 1.0000000 goodnight beautiful worldsweet dreams olive juice goodnight beautiful worldsweet dreams olive juice FALSE 53 9 2 0 0 0 0 0 FALSE 53 9 2 0 0 0 0 0 TRUE
88c05add09 _Gamefan Thanks I love &quot;Misery Business&quot;; Paramore is such a great band! love positive 20 23 0.0769231 gamefan thanksi love quotmisery businessquot paramore is such a great band love FALSE 83 20 7 0 0 0 0 0 FALSE 4 1 0 0 0 0 0 0 FALSE
eaa0b85855 Yay for St Louis traffic Yay for St Louis traffic neutral 1 24 1.0000000 yay for st louis traffic yay for st louis traffic FALSE 24 5 0 0 0 0 0 0 FALSE 24 5 0 0 0 0 0 0 TRUE
ab56747ae1 I have to pee. I have to pee. neutral 1 14 1.0000000 i have to pee i have to pee FALSE 14 5 1 0 0 0 0 0 FALSE 14 5 1 0 0 0 0 0 TRUE
b57188597f hhahaa! its well strange just woke up to an empty house ahah lolllllyyyyyyyy&lt;3 love that dog hhahaa! its well strange just woke up to an empty house ahah lolllllyyyyyyyy&lt;3 love th neutral 1 93 0.6250000 hhahaaits well strange just woke up to an empty house ahahlolllllyyyyyyyylt love that dog hhahaaits well strange just woke up to an empty house ahahlolllllyyyyyyyylt love th FALSE 99 21 3 1 0 0 0 0 FALSE 93 20 3 1 0 0 0 0 FALSE
7e4579c8f9 awesome! I am at 1505 west lake street: right across from the dunn brothers on lake awesome! positive 1 8 0.0625000 awesome i am atwest lake street right across from the dunn brothers on lake awesome FALSE 83 18 2 4 0 0 0 0 FALSE 8 2 1 0 0 0 0 0 FALSE
f20fecf55e i got followed by one that says she doesn’t talk to &quot;god people&quot;-i wonder if she talks to &quot;goddess people?&quot; i got followed by one that says she doesn’t talk to &quot;god people&quot;-i wonder if she talks to &quot;god neutral 1 109 0.6956522 i got followed by one that says she doesnt talk to quotgod peoplequoti wonder if she talks to quotgoddess peoplequot i got followed by one that says she doesnt talk to quotgod peoplequoti wonder if she talks to quotgod FALSE 127 34 11 0 0 0 0 0 FALSE 109 29 8 0 0 0 0 0 FALSE
f6ca88db6f http://twitpic.com/66r0u - uh wait…i thought this was the 29th…tonight uh wait…i thought this was the 29th…tonight neutral 28 74 0.7777778 uh waiti thought this was the thtonight FALSE 74 17 12 5 1 0 0 0 FALSE 47 15 6 2 0 0 0 0 FALSE
067058825c Will this rain ever go away, my puppy is getting cabin fever and she doesn’t like going out in the rain doesn’t like negative 70 81 0.0952381 will this rain ever go away my puppy is getting cabin fever and she doesnt like going out in the rain doesnt like FALSE 103 22 2 0 0 0 0 0 FALSE 12 2 1 0 0 0 0 0 FALSE
b084bce92a Great meeting…lunch on the patio in the sunshine…nothing more perfect busy day…time for bed, ahhhhh Great meeting.. positive 1 15 0.0588235 great meetinglunch on the patio in the sunshinenothing more perfectbusy daytime for bed ahhhhh great meeting FALSE 106 28 10 0 0 0 0 0 FALSE 15 4 2 0 0 0 0 0 FALSE
2ef8bef8cf &quot;We did it we did it oh yea yea yea no eating here tonite woo eating here tonight no no no eating here tonite you on a diet&quot; finding nemo &quot;We did it we did it oh yea yea yea no eating here tonite woo eating here tonight no no no eating here tonite you on a diet&quot; findi neutral 1 140 0.3720930 quotwe did it we did it oh yea yea yea no eating here tonite woo eating here tonight no no no eating here tonite you on a dietquot finding nemo quotwe did it we did it oh yea yea yea no eating here tonite woo eating here tonight no no no eating here tonite you on a dietquot findi FALSE 147 36 4 0 0 0 0 0 FALSE 140 35 4 0 0 0 0 0 FALSE
aa4412f02c hey adt, guess what. my princelple’s number plate is adt�000 well its not 000, i just dunno the numbers. hey adt, guess what. my princelple’s number plate is adt�000 well its not 000, i just dunno the numbers. neutral 1 106 1.0000000 hey adt guess what my princelples number plate is adtï well its not i just dunno the numbers hey adt guess what my princelples number plate is adtï well its not i just dunno the numbers FALSE 106 26 6 6 0 0 0 0 FALSE 106 26 6 6 0 0 0 0 TRUE


Check if there is an empty line:

train_metadata %>% 
  filter(text_na | sel_text_na) %>% 
  kable2(height = NULL)
textID text sel_text sentiment start end jaccard text_clean sel_text_clean text_na text_len text_n_words text_npunc text_numbers text_links text_hashtags text_retweet text_atpeople sel_text_na sel_text_len sel_text_n_words sel_text_npunc sel_text_numbers sel_text_links sel_text_hashtags sel_text_retweet sel_text_atpeople equal_texts
fdb77c3752 NA NA neutral NA NA 1 NA NA TRUE NA 0 NA NA NA NA NA NA TRUE NA 0 NA NA NA NA NA NA NA


There are 1 empty lines, let’s remove them:

train_metadata <- train_metadata %>% filter(!text_na)

Exploratory data analysis

Another kernel will be developed to make an appropriate exploratory analysis of texts and metadata. For now we are only looking for an overview of the data.

Let’s see:

train_metadata %>% 
  select(sentiment, jaccard, text_len:text_atpeople) %>% 
  gather(key, value, -sentiment) %>% 
  ggplot(aes(x = key, y = value, fill = sentiment))+
  geom_half_boxplot(side = "l", center = TRUE, errorbar.draw = FALSE)+
  geom_half_violin(side = "r")+
  facet_wrap(~key, scales = "free")+
  theme(legend.position = "botton")

Some insights after analyzing the metadata:

Baseline model

Estimated score of jaccard similarity in training data using the full text:

print(glue::glue("Jaccard similarity in training data: {round(mean(train_metadata$jaccard), 4)}"))
## Jaccard similarity in training data: 0.5475

Save the submission file using full textcolumn to obtain a reference jaccard metric as starting point for the development of more complex models:

read_csv("data/test.csv") %>% 
  select(textID, selected_text = text) %>% 
  write_csv("data/baseline.csv")

Yeahh!! Lets go submmit!